{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"..\")\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = np.array([[1, 0, 0, 0, 0, 0, 0]])\n",
    "W = np.random.randn(7, 3)\n",
    "h = np.dot(c, W)\n",
    "print(h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common.layers import MatMul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer = MatMul(W)\n",
    "h = layer.forward(c)\n",
    "print(h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample context data\n",
    "c0 = np.array([[1, 0, 0, 0, 0, 0, 0]])\n",
    "c1 = np.array([[0, 0, 1, 0, 0, 0, 0]])\n",
    "\n",
    "# initialize weight\n",
    "W_in = np.random.randn(7, 3)\n",
    "W_out = np.random.randn(3, 7)\n",
    "\n",
    "# create layer\n",
    "in_layer0 = MatMul(W_in)\n",
    "in_layer1 = MatMul(W_in)\n",
    "out_layer = MatMul(W_out)\n",
    "\n",
    "# forward propagation\n",
    "h0 = in_layer0.forward(c0)\n",
    "h1 = in_layer1.forward(c1)\n",
    "h = 0.5 * (h0 + h1)\n",
    "s = out_layer.forward(h)\n",
    "\n",
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common.util import preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"You say goodbye and I say hello.\"\n",
    "corpus, word_to_id, id_to_word = preprocess(text)\n",
    "print(corpus)\n",
    "print(id_to_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_contexts_target(corpus, window_size=1):\n",
    "    target = corpus[window_size:-window_size]\n",
    "    contexts = []\n",
    "    \n",
    "    for idx in range(window_size, len(corpus) - window_size):\n",
    "        cs = []\n",
    "        for t in range(-window_size, window_size + 1):\n",
    "            if t == 0:\n",
    "                continue\n",
    "            cs.append(corpus[idx + t])\n",
    "        contexts.append(cs)\n",
    "    \n",
    "    return np.array(contexts), np.array(target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contexts, target = create_contexts_target(corpus, window_size=1)\n",
    "print(contexts)\n",
    "print(target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common.util import create_contexts_target, convert_one_hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"You say goodbye and I say hello.\"\n",
    "corpus, word_to_id, id_to_word = preprocess(text)\n",
    "contexts, target = create_contexts_target(corpus, window_size=1)\n",
    "\n",
    "vocab_size = len(word_to_id)\n",
    "target = convert_one_hot(target, vocab_size)\n",
    "\n",
    "contexts = convert_one_hot(contexts, vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common.layers import MatMul, SoftmaxWithLoss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleCBOW:\n",
    "    def __init__(self, vocab_size, hidden_size):\n",
    "        V, H = vocab_size, hidden_size\n",
    "        \n",
    "        # initialize weights\n",
    "        W_in = 0.01 * np.random.randn(V, H).astype(\"f\")\n",
    "        W_out = 0.01 * np.random.randn(H, V).astype(\"f\")\n",
    "        \n",
    "        # create layers\n",
    "        self.in_layer0 = MatMul(W_in)\n",
    "        self.in_layer1 = MatMul(W_in)\n",
    "        self.out_layer = MatMul(W_out)\n",
    "        self.loss_layer = SoftmaxWithLoss()\n",
    "        \n",
    "        # gather all weights and gradients\n",
    "        layers = [self.in_layer0, self.in_layer1, self.out_layer]\n",
    "        self.params, self.grads = [], []\n",
    "        for layer in layers:\n",
    "            self.params += layer.params\n",
    "            self.grads += layer.grads\n",
    "        \n",
    "        # set distributed representation of words to member variables\n",
    "        self.word_vecs = W_in\n",
    "    \n",
    "    def forward(self, contxts, target):\n",
    "        h0 = self.in_layer0.forward(contexts[:, 0])\n",
    "        h1 = self.in_layer1.forward(contexts[:, 1])\n",
    "        h = (h0 + h1) * 0.5\n",
    "        score = self.out_layer.forward(h)\n",
    "        loss = self.loss_layer.forward(score, target)\n",
    "        return loss\n",
    "    \n",
    "    def backward(self, dout=1):\n",
    "        ds = self.loss_layer.backward(dout)\n",
    "        da = self.out_layer.backward(ds)\n",
    "        da *= 0.5\n",
    "        self.in_layer1.backward(da)\n",
    "        self.in_layer0.backward(da)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common.trainer import Trainer\n",
    "from common.optimizer import Adam\n",
    "from simple_cbow import SimpleCBOW\n",
    "from common.util import preprocess, create_contexts_target, convert_one_hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "window_size = 1\n",
    "hidden_size = 5\n",
    "batch_size = 3\n",
    "max_epoch = 1000\n",
    "\n",
    "text = \"You say goodbye and I say hello.\"\n",
    "corpus, word_to_id, id_to_word = preprocess(text)\n",
    "\n",
    "vocab_size = len(word_to_id)\n",
    "contexts, target = create_contexts_target(corpus, window_size)\n",
    "target = convert_one_hot(target, vocab_size)\n",
    "contexts = convert_one_hot(contexts, vocab_size)\n",
    "\n",
    "model = SimpleCBOW(vocab_size, hidden_size)\n",
    "optimizer = Adam()\n",
    "trainer = Trainer(model, optimizer)\n",
    "\n",
    "trainer.fit(contexts, target, max_epoch, batch_size)\n",
    "trainer.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vecs = model.word_vecs\n",
    "for word_id, word in id_to_word.items():\n",
    "    print(word, word_vecs[word_id])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
